<?php
/**
 * @author Aleske A. Evdokimov <ale@izhcat.ru>
 * @version $Id: repearser.php 12 2009-04-23 11:44:18Z workales $
 */

define('CSV_SEPARATOR', ';');
define('CSV_ESCAPE', '"');
define('CSV_LINEBREAK', "\n");

function untag($str) {
  return trim(preg_replace(array('`<[^>]+>`is'), '', $str));
}

function strip($str) {
  return trim(preg_replace(array('`<[^>]+>`is', '`&nbsp;`is', '`\s\s+`is'), ' ', $str));
}

function _trim($str) {
  return trim(preg_replace(array('`&nbsp;`is', '`\s\s+`is'), ' ', $str));
}

function parse_chunk($chunk, $row_format, $skip_first, $skip_last, $field_format, $fields_num, $skip_fields, $row_spans, $col_spans) {
  $tr = array();

  if (!$row_format) $row_format = '`(<tr.+?</tr>)`is';

  preg_match_all($row_format, $chunk, $tr);

  if ($skip_last > 0) $tr = array_slice($tr[1], $skip_first, -$skip_last);
  else $tr = array_slice($tr[1], $skip_first);

  if (count($tr)) {
    $lines = process_chunk($tr, $field_format, $fields_num, $row_spans, $col_spans);

    foreach ($lines as $k => $line) {
      foreach ($skip_fields as $s) array_splice($line, $s, 1);

      $lines[$k] = $line;
    }

    return $lines;
  } else return false;
}

function process_chunk($tr, $item_regex, $columns, $row_spans, $col_spans) {
  $lines_num = count($tr);
  $lines = array_fill(0, $lines_num, array_fill(0, $columns, false));

  for ($i = 0; $i < $lines_num; $i++) {

/*
Freq.   | Logo          | Provider Name  |    | Video        | SR-FEC   | NID-TID  | Beam   | Source
Tp      |               | Channel Name   |    | Encryption   | SID-VPID | Audio    |        | Updated
-----------------------------------------------------------------------------------------------------
10974 H | <a><img></a>  | STS (+2h)      | XY | DVB          | 8150-3/4 | 1326-801 | Wide   | Artemka
tp B1   |               |                |    | Viaccess 2.6 | 1 - 4194 | 4195 R   | Europe | 080810
        ----------------------------------------------------------------------------        |
        |               | Free-X Delight | Z  | BISS         |  1 |  35 | 33 R     |        |

rowspan? rowspan?                         skip colspan?      colspan?              rowspan?   rowspan?
*/

    $td = array();

    if ($item_regex) {
      preg_match_all($item_regex, $tr[$i], $td);
      $td = $td[1];

      for ($j = 0; $j < $columns; $j++) if (array_key_exists($j, $td)) $lines[$i][$j] = $td[$j];
    } else {
      preg_match_all('`(<td.+?</td>)`is', $tr[$i], $td);

      foreach ($td[1] as $d) {
        $s = array();

        if ($row_spans && preg_match('`(.*)rowspan=[\'"]?(\d+)[\'"]?(.*)`is', $d, $s)) {
          $row_span = (int)$s[2];
          $d = $s[1] . $s[3];
        } else $row_span = 1;

        if ($col_spans && preg_match('`(.*)colspan=[\'"]?(\d+)[\'"]?(.*)`is', $d, $s)) {
          $col_span = (int)$s[2];
          $d = $s[1] . $s[3];
        } else $col_span = 1;

        $l = 0;
        while (($l < $columns) && ($lines[$i][$l] !== false)) $l++;
        for ($k = $i; $k < $i + $row_span; $k++) {
          for ($j = $l; $j < $l + $col_span; $j++) $lines[$k][$j] = $d;
        }
      }
    }
  }

  return $lines;
}

function process_source($str, $START, $END, $marker_start, $marker_end) {
  $str = stristr($str, $START);
  $str = substr($str, strlen($START), strripos($str, $END) - strlen($START));

  $chunks = array();
  $temp = array();
  while (preg_match($marker_start, $str, $temp, PREG_OFFSET_CAPTURE)) {
    $chunk_start = $temp[1][1];
    preg_match($marker_end, $str, $temp, PREG_OFFSET_CAPTURE, $chunk_start);
    $chunk_end = $temp[1][1] + strlen($temp[1][0]);
    $chunks[] = substr($str, $chunk_start, $chunk_end - $chunk_start);
    $str = substr($str, $chunk_end);
  }

  return $chunks;
}

// fputcsv() suxx
function make_csv($data, $header, $names) {
  $r = array();

  if ($header) $r[] = implode(CSV_SEPARATOR, $names);

  foreach ($data as $line) {
    $o = array();
    foreach ($names as $i) {
      $item = $line[$i];
      if (stripos($item, CSV_ESCAPE) !== false) $item = str_replace(CSV_ESCAPE, CSV_ESCAPE . CSV_ESCAPE, $item);
      if ((stripos($item, CSV_SEPARATOR) !== false)
        || (stripos($item, CSV_ESCAPE) !== false)
        || (stripos($item, CSV_LINEBREAK !== false))) $item = CSV_ESCAPE . $item . CSV_ESCAPE;
      $o[] = $item;
    }
    $r[] = implode(CSV_SEPARATOR, $o);
  }

  return implode(CSV_LINEBREAK, $r) . CSV_LINEBREAK;
}

function find_value($src, $definition) {
  $source = '';

  if (array_key_exists('src', $definition)) {
    if (is_array($src) && array_key_exists($definition['src'], $src)) $source = $src[$definition['src']];
  } else $source = $src;

  $kind = 'plain';
  if (array_key_exists('kind', $definition)) $kind = $definition['kind'];

  $ret = array_key_exists('multiple', $definition) ? array() : '';

  switch ($kind) {
    case 'regex' :
      $s = array();

      if (array_key_exists('multiple', $definition)) {
        if (preg_match_all($definition['regex'], $source, $s)) {
          if (array_key_exists(1, $s)) $ret = $s[1];
        }
      } else {
        if (preg_match($definition['regex'], $source, $s)) {
          if (array_key_exists('found', $definition)) $ret = $definition['found'];
          else {
            if (array_key_exists(1, $s)) $ret = $s[1];
          }
        }
      }
    break;
    case 'list' :
      $source = strip($source);

      if (array_key_exists('multiple', $definition)) {
        foreach ($definition['list'] as $e) {
          $pos = stripos($source, $e);
          if ($pos !== false) {
            $ret[] = $e;
            $source = str_ireplace($e, '', $source);
          }
        }
      } else {
        foreach ($definition['list'] as $e) {
          $pos = stripos($source, $e);
          if ($pos !== false) {
            if (array_key_exists('found', $definition)) $ret = $definition['found'];
            else $ret = $e;
            break;
          }
        }
      }
    break;
    default :
      $ret = $source;
  }

  if (array_key_exists('multiple', $definition)) {
    if (array_key_exists('untag', $definition)) foreach ($ret as $k => $r) $ret[$k] = untag($r);
    if (array_key_exists('strip', $definition)) foreach ($ret as $k => $r) $ret[$k] = strip($r);
    if (array_key_exists('trim', $definition)) foreach ($ret as $k => $r) $ret[$k] = _trim($r);

    $ret = array_filter($ret);
    $ret = count($ret) ? implode($definition['default'], $ret) : '';
  } else {
    if (array_key_exists('untag', $definition)) $ret = untag($ret);
    if (array_key_exists('strip', $definition)) $ret = strip($ret);
    if (array_key_exists('trim', $definition)) $ret = _trim($ret);
  }

  if (array_key_exists('mandatory', $definition) && ($ret == '')) return false;

  if (array_key_exists('skip', $definition)) {
    if ($ret != '') return false;
    if (array_key_exists('found', $definition)) $ret = $definition['found'];
  }

  if (!array_key_exists('multiple', $definition) && ($ret === '') && array_key_exists('default', $definition)) $ret = $definition['default'];

  return (string)$ret;
}

function help($def = false) {
  global $argv;

  if ($def) {
    echo <<<EOT
Syntax of DEFINITION.def:

  Each .def file can include Options and field Definitions, which can be Non-repeatable
  and Repeatable.

  Option is described in single line as

    ! option_name {'option value'|keyword|number}

  Non-repeatable and Repeatable Definitions described on three lines as

    {-|number} 'FieldName' {'default value'|keyword}
    field_type 'value'
    {flag[ flag]...[ 'found_value']|default}

  Strings beginning with # are comments

    #this is a comment

Options:

  scope_start
    Substring which starts the search region
  Example:
    ! scope_start '<!-- big table start -->'
  By default:
    ! scope_start '<html>'

  scope_end
    Substring which ends the search region
  Example:
    ! scope_end '<!-- big table end -->'
  By default:
    ! scope_end '</html>'

  chunk_start
    Regexp which indicates beginning of meaningful data chunk
  Example:
    ! chunk_start '`(<table.{0,400}?width=720)`is'
  By default:
    ! chunk_start '`(<table>)`is'

  chunk_end
    Regexp which indicates end of meaningful data chunk
  Example:
    ! chunk_end '`(colspan=10.{0,200}?</table>)`is'
  By default:
    ! chunk_end '`(</table>)`is'

  row_format
    Regexp which indicates single row of data in the chunk, or keyword html_table.
    If this option is set to some other value, then field_format option MUST be set
  Example:
    ! row_format '`(<div.+?id="row".+?</span></div>)`is'
  By default:
    ! row_format html_table which is equal to '`(<tr.+?</tr>)`is', but also enables
    processing of colspans and rowspans

  field_format
    Regexp which indicates single field of data in the row, works ONLY in conjunction
    with row_format. Also can be html_table
  Example:
    ! field_format '`(<span.+?id="field-\d+".+?</span>)`is'
  By default:
    ! field_format html_table which is '`(<td.+?</td>)`is' plus processing of rowspans
    and colspans

  row_spans
    Number. If it is equals to 0, then disables processing of rowspans for row_format
    and field_format html_table
  Example:
    ! row_spans 0
  By default
    ! row_spans 1

  col_spans
    Number. If it is equals to 0, then disables processing of colspans for row_format
    and field_format html_table
  Example:
    ! col_spans 0
  By default
    ! col_spans 1

  skip_first
    Number which sets the amount of rows to skip on beginning of each chunk
  Example:
    ! skip_first 2
  By default:
    ! skip_first 0

  skip_last
    Number which sets the amount of rows to skip on end of each chunk
  Example:
    ! skip_last 1
  By default:
    ! skip_last 0

  fields_num
    Number which sets the amount of fields in each row of data
  Example:
    ! fields_num 9
  By default:
    ! fields_num 0

  skip_fields
    Semicolon delimited list of fields to skip in each row of data. Fields are numbered from
    zero to (fields_num - 1)
  Example:
    ! skip_fields '3;9'
  By default:
    No fields are skipped

Definitions:

  {-|number} 'FieldName' {'default value'|keyword}
  field_type 'value'
  {flag[ flag]...[ 'found_value']|default}

    {-|number}
      Index of field for Repeatable fields, or - (hyphen ) for Non-repeatables.
      NB! Non-repeatable fields are searched in the ENTIRE file, regardless of chunk_start and
      chunk_end options

    'FieldName'
      String which is name of data field

    {'default value'|keyword}
      String for the default value or glue for multiple fields (see below), or one of the
      following keywords

        mandatory, which leads to skip data row if the value is not found,

        skip for the row to be skipped if it was found,

        unique for the only first row with found value to be included in output.

      NB! If the non-repeatable field is mandatory and not found, or is skippable and found,
      then the ENTIRE file is NOT processed.

    field_type
      Three field types are recognized by the folloeing keywords: regex, list and plain text.
        regex
          Regular expression can capture any number of matches, but only the first is returned
          by the parser. If the regular expression does not capture anything, but matches,
          then the 'found_value' (see below) is returned
        list
          Semicolon delimited list (dictionary) of possible values. If the 'found_value' is set,
          then it will be returned instead of actually found value
        plain text
          The field data is returned as is
      NB! Please note the plain text keyword is actually two keywords, and second of it (text)
      cannot be omitted

    'value'
      Search value as string
    Examples:
      regex '`bgcolor="#ffffff"|bgcolor="#ffdddd"`is'
      regex '`<font\s+size=2><b>(\S+)`is'
      regex '`<b>.*?\d+\s+(\S+).*?</b>`is'
      list '16-QAM;16APSK;32APSK;8PSK;MAPSK;QPSK'

    flags
      untag
        Strip all the HTML from returned value
      strip
        Strip all the HTML from returned value and replace multiple whitespace characters with
        single whitespace
      trim
        Strip whitespaces on left and right sides of returned value
      multiple
        Treat the value as array, and glue it before return into single string with
        'default value' string
      silent
        Found value won't be included in output

    'found_value'
      Return this string instead of the actually found data

    default
      This keyword MUST be set, if no flags or 'found_value' are required

Notes on definitions:

  For fields that are simultaneously mandatory and silent 'found_value' MUST be set, otherwise
  they cannot be found at all. Just use any dummy value like ''.

  The unique keyword implies mandatory.

  Unique field CAN be multiple.

  If more of than one of the untag, strip, and trim flags are set, they's operations will be
  performed on found data in the order as listed here. No flags conflicts with each other

EOT;
  } else {
    echo <<<EOT
Usage:
  php $argv[0] --def
  php $argv[0] FILENAME.html DEFINITION.def [--no-header] > OUTFILENAME.ext

Parameters:

  FILENAME.html
    Input file name

  DEFINITION.def
    Definition file name

Switches:
  --def[-syntax] show syntax of .def file and exit
  --no-head[er]  supresses header in output CSV or HTML

Examples:
  php $argv[0] http://www.lyngsat.com/arab2b.html ./sat_000_tp.def > arab2b_transponders.csv
  php $argv[0] ./testdata/badr.html ./sat_005_chan_tv.def --ho-header > badr_tv.csv

EOT;
  }

  exit(1);
}

$cmd = implode(' ', array_slice($argv, 1));
if (stripos($cmd, '--def') !== false) help(true);

if ($argc < 3) help();

$cmd = implode(' ', array_slice($argv, 3));
$no_header = (stripos($cmd, '--no-head') !== false);

$refname = $argv[1];

@$F = fopen($refname, "r");
if (!$F) help();

$defname = $argv[2];

@$D = fopen($defname, "r");
if (!$D) help();
fclose($D);

$str = stream_get_contents($F);
fclose($F);

$def = file($defname);

$options = array();
$definitions = array();

$silents = array();
$uniques = array();

$i = 0;
foreach ($def as $d) {
  $d = trim($d);
  if (($d == '') || ($d[0] == '#')) continue;

  switch ($i) {
    case 0 :
      $d = explode(' ', $d, 2);

      if ($d[0] == '!') {
        $o = explode(' ', $d[1], 2);
        $options[$o[0]] = $o[1];

        $i = 0;
      } else {
        $o = explode(' ', $d[1], 2);

        $name = trim($o[0], "'");

        $dd = array('name' => $name);

        if ($o[1] == 'mandatory') {
          $dd['mandatory'] = true;
        } elseif ($o[1] == 'unique') {
          $dd['mandatory'] = true;
          $dd['unique'] = true;
          $uniques[$name] = array();
        } elseif ($o[1] == 'skip') {
          $dd['skip'] = true;
        } else {
          $dd['default'] = trim($o[1], "'");
        }

        if ($d[0] == '-') {
          $dd['repeated'] = true;
        } else {
          $dd['src'] = (int)$d[0];
        }

        $i = 1;
      }
    break;
    case 1 :
      $d = explode(' ', $d, 2);

      switch ($dd['kind'] = $d[0]) {
        case 'regex' :
          $dd['regex'] = trim($d[1], "'");
        break;
        case 'list' :
          $dd['list'] = explode(';', trim($d[1], "'"));
        break;
      }

      $i = 2;
    break;
    case 2 :
      $d = explode(' ', $d);

      foreach ($d as $flag) {
        switch ($flag) {
          case 'trim' : $dd['trim'] = true; break;
          case 'strip' : $dd['strip'] = true; break;
          case 'untag' : $dd['untag'] = true; break;
          case 'multiple' : $dd['multiple'] = true; break;
          case 'silent' : $silents[] = $name; break;
          case 'default' : break;
          default : $dd['found'] = trim($flag, "'");
        }
      }

      $definitions[$dd['name']] = $dd;

      $i = 0;
    break;
  }
}

$stop = false;
foreach ($definitions as $k => $definition) {
  if (array_key_exists('repeated', $definition)) {
    $val = find_value($str, $definition);

    if ($val === false) {
      $stop = true;
      break;
    } else $definitions[$k]['value'] = $val;
  }
}

if ($stop) exit(2);

$scope_start = array_key_exists('scope_start', $options) ? trim($options['scope_start'], "'") : '<html>';
$scope_end = array_key_exists('scope_end', $options) ? trim($options['scope_end'], "'") : '</html>';

$chunk_start = array_key_exists('chunk_start', $options) ? trim($options['chunk_start'], "'") : '<table>';
$chunk_end = array_key_exists('chunk_end', $options) ? trim($options['chunk_end'], "'") : '</table>';

$row_format = false;
if (array_key_exists('row_format', $options)) {
  if ($options['row_format'] != 'html_table') $row_format = trim($options['row_format'], "'");
}

$skip_first = array_key_exists('skip_first', $options) ? (int)$options['skip_first'] : 0;
$skip_last = array_key_exists('skip_last', $options) ? (int)$options['skip_last'] : 0;

$field_format = false;
if ($row_format) {
  if (array_key_exists('field_format', $options)) {
    if ($options['field_format'] != 'html_table')
      $field_format = trim($options['field_format'], "'");
  } else help(true);
}

$fields_num = array_key_exists('fields_num', $options) ? (int)$options['fields_num'] : 0;
$skip_fields = array();
if (array_key_exists('skip_fields', $options)) {
  foreach (split(';', trim($options['skip_fields'], "'")) as $sk) {
    $sk = (int)$sk;
    if (($sk < $fields_num) && !in_array($sk, $skip_fields)) $skip_fields[] = $sk;
  }
}

$row_spans = array_key_exists('row_spans', $options) ? (bool)(int)$options['row_spans'] : true;
$col_spans = array_key_exists('col_spans', $options) ? (bool)(int)$options['col_spans'] : true;

$chunks = process_source($str, $scope_start, $scope_end, $chunk_start, $chunk_end);

$i = 0;
$data = array();
$raw_data = array();

foreach ($chunks as $chunk)
  if ($parsed = parse_chunk($chunk, $row_format, $skip_first, $skip_last, $field_format, $fields_num, $skip_fields, $row_spans, $col_spans))
    $raw_data = array_merge($raw_data, $parsed);

foreach ($raw_data as $src) {
  $line = array();

  $val = false;

  foreach ($definitions as $definition) {
    $name = $definition['name'];

    if (array_key_exists('repeated', $definition)) $line[$name] = $definition['value'];
    else {
      $val = find_value($src, $definition);

      if (array_key_exists($name, $uniques) && ($val !== false)) {
        if (in_array($val, $uniques[$name])) $val = false;
        else $uniques[$name][] = $val;
      }

      if ($val === false) break;
      else $line[$name] = $val;
    }
  }

  if ($val === false) continue;

  $data[] = $line;
}

if (count($data))
  echo iconv('ISO8859-1', 'UTF-8', make_csv($data, !$no_header, array_diff(array_keys($definitions), $silents)));

exit(0);
